{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.5605252,  0.       ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('MountainCarContinuous-v0', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "\n",
    "        reward = -1.0\n",
    "        if done:\n",
    "            reward = 100\n",
    "\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhr0lEQVR4nO3daWwTd/4/8Pc4vnLZDjlsQhwCBRood06TQEtISVm2lFJtd7t9UHVXrWjDqrSoUiP9S7XSSqm2T/aq6EqrpX2wWlaslt0WQdqQQA7iHA6kmISG0kITGjsBktg5HMf2fP4P2swPA6UJhIyPz0saqZ752v54wrw7852Z7whERGCMMRko5C6AMRa9OIAYY7LhAGKMyYYDiDEmGw4gxphsOIAYY7LhAGKMyYYDiDEmGw4gxphsOIAYY7KRLYDef/99ZGVlQavVoqCgAK2trXKVwhiTiSwB9K9//QtvvPEG3nnnHZw5cwZr1qxBWVkZBgYG5CiHMSYTQY6bUQsKCpCXl4e//OUvAABRFGE2m/Gb3/wGb7311lyXwxiTiXKuv3BychLt7e2oqKiQ5ikUCpSWlsJqtd7xPV6vF16vV3otiiIGBweRnJwMQRAeeM2MsZkhIoyMjCA9PR0KxQ8faM15AF2/fh2BQABGozFovtFoxBdffHHH91RWVuK3v/3tXJTHGJtFvb29yMjI+MHlcx5A96KiogJvvPGG9NrlciEzMxO9vb3Q6XQyVsYYuxO32w2z2YzExMS7tpvzAEpJSUFMTAz6+/uD5vf398NkMt3xPRqNBhqN5rb5Op2OA4ixEPZjXSRzfhZMrVYjJycHNTU10jxRFFFTUwOLxTLX5TDGZCTLIdgbb7yBF154Abm5ucjPz8cf/vAHjI2N4cUXX5SjHMaYTGQJoJ///Oe4du0a9u/fD6fTibVr16Kqquq2jmnGWGST5Tqg++V2u6HX6+FyubgPiLEQNN1tNCzOgjHGHrzJyW8RCAxBpTIjJiYOgPKBX2fHAcQYAwAMD/8XfX3/D0plGrTabMTFrUFs7GpotY9ApTIiJkaPqfNWsxVMHECMse+JCASGEQgMw+u9CJfrYwBKKBRaqNVmaDRLEB+fD52uDPHxebPyjRxAjLEfoIBSaYBSOR+xsY9Ie0QqVfqsfQMHEGMMAODzKaHVrkVs7PLvw2YdNJpFUKnSoVBoAShmvU+IA4gxBgA4cyYVGzb8D/PmLYAgxMzJd/KIiIwxAIDPp4AgJMxZ+AAcQIwxGXEAMcZkwwHEGJMNBxBjTDYcQIwx2XAAMcZkwwHEGJMNBxBjTDYcQIwx2XAAMcZkw/eCMcZ+1M0DpxJR0E2p93ODKgcQYywIEcHtdmN0dBQOhwPXrl3DxMQEXC4XYmNjMTo6itjYWKhUKmi1WsybNw8JCQlYsGABEhMToVarp/1dHECMRTkigs/nw+DgID799FOIogiPxwOTyYTU1FQsWbIESUlJUKlUQY9ZFkURExMTGBwchMPhwOXLl+FyuRAfH4+kpKRpfTcHEGNRiIggiiIcDgfOnj2L69evQ6vVIicnB5mZmUEPAr3bIZZer4fRaER2drY0b3h4GN3d3dOqgwOIsSjj9Xpx8eJF2Gw2mEwmrF+/HmlpaVAq730Q+pvfl5SUhBUrVkzrfRxAjEWBqcMsu92O8+fPIzs7Gz/72c+QkJAga10cQIxFuEAggM7OTtjtdixfvhzPPfccVCrVA3/kznRwADEWoYgI3377Ldra2pCSkoKdO3ciLi4uJIJnCgcQYxFocnISra2t6O/vR0lJCQwGQ0gFzxQOIMYiCBGhr68PJ0+eRG5uLiwWC2Ji5m6M55niAGIsQgQCAXR0dKC/vx87duxAYmJiSO713GzG94LV19fjySefRHp6OgRBwH//+9+g5USE/fv3Y/78+YiNjUVpaSm+/PLLoDaDg4N4/vnnodPpYDAY8Otf/xqjo6P39UMYi2ZjY2P49NNPIQgCysrKoNPpQj58gHsIoLGxMaxZswbvv//+HZf//ve/x5/+9Cd88MEHaGlpQXx8PMrKyjAxMSG1ef7559HZ2Ynq6mocPXoU9fX1ePnll+/9VzAWpYgIvb29OHHiBNatW4d169aF9CHXbeg+AKAjR45Ir0VRJJPJRO+99540b3h4mDQaDf3zn/8kIqKuri4CQG1tbVKb48ePkyAI9O23307re10uFwEgl8t1P+UzFtZEUaQvvviC/v3vf9Po6CiJoih3SZLpbqOzOhzH5cuX4XQ6UVpaKs3T6/UoKCiA1WoFAFitVhgMBuTm5kptSktLoVAo0NLScsfP9Xq9cLvdQRNj0czv98NqteLatWvYsWMH4uPjw+KQ61azGkBOpxMAYDQag+YbjUZpmdPpRFpaWtBypVKJefPmSW1uVVlZCb1eL01ms3k2y2YsrPh8PjQ0NEClUqGoqAgqlUruku5ZWAxIVlFRAZfLJU29vb1yl8TYnCMiTExMoK6uDpmZmcjNzQ3LvZ6bzWoAmUwmAEB/f3/Q/P7+fmmZyWTCwMBA0HK/34/BwUGpza00Gg10Ol3QxFg0ISJ4vV5UV1fj4YcfxuLFi8M+fIBZDqBFixbBZDKhpqZGmud2u9HS0gKLxQIAsFgsGB4eRnt7u9SmtrYWoiiioKBgNsthLGKMjY2htrYWhYWFyMjIiIjwAe7hQsTR0VFcunRJen358mV0dHRg3rx5yMzMxN69e/G73/0OS5cuxaJFi/D2228jPT0dO3fuBAAsX74cTzzxBF566SV88MEH8Pl82LNnD37xi18gPT191n4YY5GAiDA6Ooqamhps2rQJ8+bNk7uk2TXT02snT54kALdNL7zwAhF9d2rw7bffJqPRSBqNhrZs2ULd3d1Bn3Hjxg167rnnKCEhgXQ6Hb344os0MjIy7Rr4NDyLFm63m44fP05DQ0MhdZr9x0x3GxWIbhptOky43W7o9Xq4XC7uD2IRiYgwPDyM06dPo7i4GAaDQe6SZmS622hYnAVjLJoQEcbHx3H69GlYLBbo9Xq5S3pg+GZUxkLMxMQETp06hQ0bNkRen88teA+IsRDi9XpRU1OD3NzcaT9ZIpxxADEWIqaucF67di3S0tIi5lT73XAAMRYCfD4fampqsHjxYixYsCAqwgfgAGJMdkSE1tZWmEwmLFq0KGrCB+AAYkxWRAS73Q61Wo01a9ZEVfgAHECMyYaIcP78efT19SEnJyfqwgfgAGJMNg6HA319fSgpKQl65no0ic5fzZjMpm7I3rRpE9RqtdzlyIYDiLE55vF4cOzYMVgsFmi1WrnLkVVYXwkdhrexsSgXCARQX1+PRx99FCkpKXKXI7uw3gNyu90cQixsiKIIq9WKjIwMHnrme2EdQE1NTZicnJS7DMZ+FBHhwoULmJycxIoVK6LyjNedhHUAZWdno6amBn6/X+5SGPtBRIT+/n709vZi48aNHD43CesAysrKQkpKCjo6OvhQjIWs0dFRWK1WbNq0KayfYPEghHUACYKAnJwcuN1ufPPNNxxCLOT4fD6cOnUKeXl5iI2NlbuckBPWAQQAMTExKC4uhs1mw9DQkNzlMCYRRRHNzc1YtmxZVN1gOhNhH0AAoFKp8Nhjj6GhoQFer1fuchiTbrNQKBRYunQph88PiIgAEgQBycnJWLVqFZqamiCKotwlsShGRLh27RouX76MgoKCqL3NYjoiZs0IgoBFixYhPj4eZ8+e5f4gJhuPx4Pm5maUlJRAqQzra30fuIgJIOC7EFq/fj2cTiecTieHEJtzfr8ftbW1WL9+PRITE+UuJ+RFVAABgFKpxObNm9HS0gKPxyN3OSyKEBFsNhtSU1OxYMECucsJCxEXQAAQGxuLvLw8nDhxAj6fT+5yWBQgIly9ehUjIyPIzc3lTudpisgAEgQB6enpSE9P5/4gNifcbjdaW1uxceNGxMTEyF1O2IjIAAK+C6F169ZhZGQEPT09cpfDIpjf70dNTQ0KCgqg0WjkLiesRGwAAd9dpFhUVASbzYbBwUHeE2KzbqrfZ9myZcjIyOBDrxmK6AACAI1GA4vFgrq6OgQCAbnLYRHmq6++wvj4OFasWCF3KWFpRgFUWVmJvLw8JCYmIi0tDTt37kR3d3dQm4mJCZSXlyM5ORkJCQl45pln0N/fH9Smp6cH27dvR1xcHNLS0vDmm28+sDvaBUHA/PnzsXTpUthsNt4LYrNmeHgYra2tKCws5IsN79GM1lpdXR3Ky8vR3NyM6upq+Hw+bN26FWNjY1Kb119/HZ988gkOHz6Muro69PX1YdeuXdLyQCCA7du3Y3JyEk1NTfjoo4/w4YcfYv/+/bP3q24hCAIeeeQRjI6O4quvvuIQYvfN5/OhsbERjz/+OOLi4uQuJ3zRfRgYGCAAVFdXR0REw8PDpFKp6PDhw1KbCxcuEACyWq1ERHTs2DFSKBTkdDqlNgcOHCCdTkder3da3+tyuQgAuVyuGdU7Pj5O//jHP8jtds/ofYzdTBRFOn36NHV2dpIoinKXE5Kmu43e136jy+UCAMybNw8A0N7eDp/Ph9LSUqlNdnY2MjMzYbVaAQBWqxWrVq2C0WiU2pSVlcHtdqOzs/OO3+P1euF2u4OmexEbG4vS0lLU19fzIGbsnhARvvzyS9y4cQPZ2dnc6Xyf7jmARFHE3r17UVRUhJUrVwIAnE4n1Go1DAZDUFuj0Qin0ym1uTl8ppZPLbuTyspK6PV6aTKbzfdaNlJTU2E2m7k/iN2TkZERdHV14fHHH+d+n1lwz2uwvLwc58+fx6FDh2aznjuqqKiAy+WSpt7e3nv+LEEQsHLlSly7dg1XrlzhEGLT5vf7UV9fj6Kioqh/nM5suacA2rNnD44ePYqTJ08iIyNDmm8ymTA5OYnh4eGg9v39/TCZTFKbW8+KTb2eanMrjUYDnU4XNN0PhUKB0tJSnDt3DqOjo/f1WSw6EBHq6+uRnp7Oj9OZRTMKICLCnj17cOTIEdTW1mLRokVBy3NycqBSqVBTUyPN6+7uRk9PDywWCwDAYrHAbrdjYGBAalNdXQ2dTjen11LExsaioKAAjY2N3B/E7oqI0NPTA7/fj7Vr13K/z2yaSc/2K6+8Qnq9nk6dOkUOh0OaxsfHpTa7d++mzMxMqq2tJZvNRhaLhSwWi7Tc7/fTypUraevWrdTR0UFVVVWUmppKFRUVs97D/mNEUaT29nY6ffo0n81gP2hkZIT+85//kMfjkbuUsDHdbXRGAQTgjtPBgwelNh6Ph1599VVKSkqiuLg4evrpp8nhcAR9zpUrV2jbtm0UGxtLKSkptG/fPvL5fLP+46bD7/fTZ599Rj09Pff9WSzyTE5O0uHDh6mvr0/uUsLKdLdRgSj8emHdbjf0ej1cLtd99wcB341gV1VVha1btyIuLo53sRmA7w692tvbpYHu+N/F9E13G+XziAC0Wi3y8/Nx4sQJHk+aSa5evYqhoSHu93mAOIDwf+MHZWRk4PPPP+dT8wxutxstLS0oKiri8X0eIA6g7wmCgLVr16K/vx8Oh0PucpiMAoEAGhsbUVhYyA8TfMA4gG4SExODTZs2obGxMegGWxY9iAgdHR2YP38+P0xwDnAA3SIuLk66Poj7g6ILEaG3txfXrl3D6tWrOXzmAAfQLQRBQGZmJpKTk2G327k/KIp4PB60tLTwuM5ziAPoDqbGk7569Sq+/fZbDqEoEAgEUF9fjw0bNiA+Pl7ucqIGB9APiImJQXFxMVpbW/n5YhGOiPD5558jLS0N6enpcpcTVTiA7kKv1yMvLw+NjY08nnSEIiL09fXh6tWrWLNmDff7zDEOoB+RkZGB5ORknDlzhg/FIpDH40FbWxu2bNnC4/vIgNf4jxAEAWvWrIHT6URfXx+HUATx+/1oaGhAfn4+4uPjee9HBhxA06BUKrFlyxbuD4og9P3zvJKTkzF//ny5y4laHEDTNDV+0KlTp3j8oAjQ29uL0dFRrFu3jvd8ZMQBNE1TzxdLTU3l582HMSLCyMgIzpw5g+LiYr7eR2YcQDMwNSyD2+3m582HqUAggGPHjqGwsJCf4x4COIBmaOp58x0dHRgaGuI9oTAiiiJaW1uxatUqmEwmPvQKARxA90Cj0aCoqAh1dXXcHxQmiAiXLl2C3+9Hdna23OWw73EA3QNBEJCcnIxHHnkEVquVb1oNcUSEgYEB2Gw2fo57iOG/xD0SBAFLlixBTEwMOjs7+VAshHm9XjQ3N+OnP/0p1Gq13OWwm3AA3QdBEJCfn4+LFy/yIGYhKhAIoKGhAevXr0diYqLc5bBbcADdJ5VKhSeeeAJtbW38kMMQMzWovMFgQEZGBnc6hyAOoFkQFxcHi8WCuro6+Hw+ucth+C58vv76a3g8Hn6iRQjjAJoFgiAgNTUVWVlZaG5u5k5pmRERbty4ga6uLmzYsIEvNgxhHECzRBAErFixAhqNhjulZebxePDpp5+iuLgYSqVS7nLYXXAAzSJBEJCbm4ve3l5cuXKFQ0gGfr8f9fX1KCkpQVJSEh96hTgOoFmmUCjw6KOP4uzZs3yl9BwTRRFNTU3IysqCyWSSuxw2DRxAD0B8fDxKS0vR2NiIiYkJucuJCkQEu92OmJgYPPzww7znEyZmFEAHDhzA6tWrodPpoNPpYLFYcPz4cWn5xMQEysvLkZycjISEBDzzzDPo7+8P+oyenh5s374dcXFxSEtLw5tvvhmRtzMkJiYiJycHVVVVmJyclLuciEZEuHLlCgYHB1FYWMjhE0ZmFEAZGRl499130d7eDpvNhpKSEjz11FPo7OwEALz++uv45JNPcPjwYdTV1aGvrw+7du2S3h8IBLB9+3ZMTk6iqakJH330ET788EPs379/dn9VCJh63PPy5cvR1NTEZ8YeECKC0+lEZ2cnP0Y5HNF9SkpKor/97W80PDxMKpWKDh8+LC27cOECASCr1UpERMeOHSOFQkFOp1Nqc+DAAdLpdOT1eqf9nS6XiwCQy+W63/IfOFEU6cyZM2Sz2UgURbnLiThut5v+97//0ejoqNylsJtMdxu95z6gQCCAQ4cOYWxsDBaLBe3t7fD5fCgtLZXaZGdnIzMzE1arFQBgtVqxatUqGI1GqU1ZWRncbre0FxVppsaUHhwcxLlz57hTehaNj4+joaEBGzdu5Gd5hakZB5DdbkdCQgI0Gg12796NI0eOYMWKFXA6nVCr1TAYDEHtjUYjnE4nAMDpdAaFz9TyqWU/xOv1wu12B03hRKFQYPPmzRgcHOTT87PE6/WiqqoKa9asue3fHAsfMw6ghx9+GB0dHWhpacErr7yCF154AV1dXQ+iNkllZSX0er00mc3mB/p9D4JSqURxcTEuXLgAp9PJIXQfAoEAGhsbsX79eixYsIA7ncPYjANIrVZjyZIlyMnJQWVlJdasWYM//vGPMJlMmJycxPDwcFD7/v5+6ZoMk8l021mxqdd3u26joqICLpdLmnp7e2dadkhQqVR47LHHYLPZcOPGDbnLCUtT4ZORkYGFCxfKXQ67T/d9HZAoivB6vcjJyYFKpUJNTY20rLu7Gz09PbBYLAAAi8UCu92OgYEBqU11dTV0Oh1WrFjxg9+h0WikU/9TU7iKi4vD5s2b0dbWhqGhIbnLCSuiKMJmsyE2NhbLli3jPZ9IMJOe7bfeeovq6uro8uXLdO7cOXrrrbdIEAT67LPPiIho9+7dlJmZSbW1tWSz2chisZDFYpHe7/f7aeXKlbR161bq6OigqqoqSk1NpYqKigfSwx6qRFGk4eFhOnToEA0ODvLZsWkIBAJks9moo6ODAoGA3OWwHzHdbXRGAfSrX/2KFi5cSGq1mlJTU2nLli1S+BAReTweevXVVykpKYni4uLo6aefJofDEfQZV65coW3btlFsbCylpKTQvn37yOfzzaSMsA+gKTdu3KCqqipyu91ylxLSAoEANTU10enTpzl8wsR0t1GBKPx6Q91uN/R6PVwuV1gfjgHA0NAQGhoaUFxczDdP3oEoirDb7ZicnEROTg6P5xwmpruN8l9TZgaDARaLBU1NTRgZGZG7nJAiiiKam5vh8/k4fCIU/0VlNjWYWVFREZqamjA4OMin6PFd+Jw5cwYAsH79eg6fCMV/1RBhMBhQUFCAEydORP0p+kAggObmZqhUKn6MToTjv2yIEAQBSUlJ2LZtG9ra2uBwOKJyT8jn86Gurg4ajQarV6/m8Ilw/NcNMQkJCdi8eTPOnDmDS5cuRVUIjY2Noba2FllZWTyQfJTgAAoxgiBAq9Vi69atcDgcOHfu3H0P5eHz+eB2u+HxeBAIBEIu1IgILpcLx48fx/Lly7Fo0SIOnyjBI3aHKJVKhaKiIrS2tuLUqVPYuHEjVCrVPX3WiRMnsG/fPqSnpyM9PR1msxmZmZlYuHAh5s+fD4PBAIPBgPj4eCgUCmnjn4sQICJ88803OHfuHEpKSjBv3rwH/p0sdHAAhbCYmBgUFhbiq6++QnV1NYqLi5GYmDjjYLhy5QouXLiACxcuSPMEQYAgCNBoNFIApaamSsFkNpuxYMECpKenY+HChUhOTp7tnwdRFNHZ2YmLFy9i+/bt0Gg0s/4dLLRxAIU4QRDw0EMPQafToaqqCoWFhTCbzTMKoY6Ojtvm0XdXwcPj8cDj8cDhcNwWUCqVCmq1Gr/85S/x17/+dTZ+jvTdXq8XjY2N0Ol0eOqpp/jxOVGK+4DCwNS1Qjt27EB3dzcaGxunPY42EWFwcHDG30lEmJycxOjo6KyO2U1EGBgYwNGjR5Geno68vDwOnyjGARQmpjqnt2zZAq1Wi48//hjXr1//0Q7lQCAgPZljaq9GpVLNaA8qPz//vmqf4vP50NDQgJaWFpSVlWH58uXc2Rzl+H89YUahUCAvLw9LlixBbW0t0tLSUFhYCKVSeceN2ev1YmhoCBqNBhaLBdnZ2RAEAd3d3Whqagp6bJBGo0FiYiL8fj9cLpcUbgsWLLivmokIDocDra2tyMjI4MHjmYQDKEwZDAY89dRTOHfuHD7++GPk5ubCbDbfduGey+XC119/jS1btiA3N1danpqaivj4eBw7dgyBQAALFy7E1q1bkZqaCp/Ph66uLtTU1GBiYgJxcXH3XOf4+Dja2trQ19eHbdu2Qa/X814Pk3AAhSlBEKBUKrFu3TosWbIEjY2NsNvtKCwsRHJysrSRBwIBpKSkYN26dUHhpFAosHr1apw7dw5utxs7duyQznSp1Wrk5OTA5/Ohvr4eWq12RrUREfx+Pz7//HNcvHgRubm5KC4u5r0edhsOoDAnCAJ0Oh22bduGgYEBVFdXIzExEQUFBUhJSYHRaMR7772H5ubm296rUqmQlZWF+Pj4266/USgUWLt2LRwOBxYvXjzterxeL+x2Oy5duoSHHnoIu3btgkaj4b0edkccQBFCEAQYjUY8++yzcDgcaGhogN/vR35+vrSnMz4+HvSe2NhYHDx4EF1dXbDZbLd9plarRUlJyY8+dYKIMDIyArvdjt7eXmRkZODJJ59EXFwcBw+7Kw6gCBMTE4MFCxZg586duHbtGux2O7788ktotVq43W7ExMRIh2+PPfYYsrKyoFAoYLfb4fV6gz5r2bJlePbZZ287TT7VOe3xeHDp0iV0dXVBoVBg6dKl2LVr14zPsrHoxQEUgaaucjYajUhLS0NxcTF6enpw/vx5OBwOXL16FevWrcPixYsxMTGBjIwMbN68GadOncLExAQUCgVMJhPKysqgUqlARPD5fPB6vRgcHMQ333yDS5cuQa1Ww2w2o6SkBCkpKXznOpsxHpI1ikxdXDgwMIBLly5haGgITqcTCQkJICKMj49jcHAQSqUS2dnZ8Pl8UCgUGBsbgyiK0Gq1iI+Px9KlS5GamgqDwcChw+5outso7wFFkal7v8xmM8xmM4gIoihCFEWMjIzA4/FIezpqtRpEBJ1Oh4SEBOmwampibDZwAEUxQRAQExODmJgYvgudyYL3nxljsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJJixvxZi6f9btdstcCWPsTqa2zR+71z0sA+jGjRsAALPZLHMljLG7GRkZgV6v/8HlYRlAUzdO9vT03PXHsWButxtmsxm9vb08jMk08Tq7N1OjZKanp9+1XVgG0NQYNHq9nv9R3AOdTsfrbYZ4nc3cdHYOuBOaMSYbDiDGmGzCMoA0Gg3eeecdaDQauUsJK7zeZo7X2YMVlmNCM8YiQ1juATHGIgMHEGNMNhxAjDHZcAAxxmQTlgH0/vvvIysrC1qtFgUFBWhtbZW7JNlUVlYiLy8PiYmJSEtLw86dO9Hd3R3UZmJiAuXl5UhOTkZCQgKeeeYZ9Pf3B7Xp6enB9u3bERcXh7S0NLz55pvw+/1z+VNk8+6770IQBOzdu1eax+tsjlCYOXToEKnVavr73/9OnZ2d9NJLL5HBYKD+/n65S5NFWVkZHTx4kM6fP08dHR30k5/8hDIzM2l0dFRqs3v3bjKbzVRTU0M2m40KCwtpw4YN0nK/308rV66k0tJSOnv2LB07doxSUlKooqJCjp80p1pbWykrK4tWr15Nr732mjSf19ncCLsAys/Pp/Lycul1IBCg9PR0qqyslLGq0DEwMEAAqK6ujoiIhoeHSaVS0eHDh6U2Fy5cIABktVqJiOjYsWOkUCjI6XRKbQ4cOEA6nY68Xu/c/oA5NDIyQkuXLqXq6mp69NFHpQDidTZ3wuoQbHJyEu3t7SgtLZXmKRQKlJaWwmq1ylhZ6HC5XAD+74bd9vZ2+Hy+oHWWnZ2NzMxMaZ1ZrVasWrUKRqNRalNWVga3243Ozs45rH5ulZeXY/v27UHrBuB1NpfC6mbU69evIxAIBP3RAcBoNOKLL76QqarQIYoi9u7di6KiIqxcuRIA4HQ6oVarYTAYgtoajUY4nU6pzZ3W6dSySHTo0CGcOXMGbW1tty3jdTZ3wiqA2N2Vl5fj/PnzaGxslLuUkNbb24vXXnsN1dXV0Gq1cpcT1cLqECwlJQUxMTG3nY3o7++HyWSSqarQsGfPHhw9ehQnT55ERkaGNN9kMmFychLDw8NB7W9eZyaT6Y7rdGpZpGlvb8fAwADWr18PpVIJpVKJuro6/OlPf4JSqYTRaOR1NkfCKoDUajVycnJQU1MjzRNFETU1NbBYLDJWJh8iwp49e3DkyBHU1tZi0aJFQctzcnKgUqmC1ll3dzd6enqkdWaxWGC32zEwMCC1qa6uhk6nw4oVK+bmh8yhLVu2wG63o6OjQ5pyc3Px/PPPS//N62yOyN0LPlOHDh0ijUZDH374IXV1ddHLL79MBoMh6GxENHnllVdIr9fTqVOnyOFwSNP4+LjUZvfu3ZSZmUm1tbVks9nIYrGQxWKRlk+dUt66dSt1dHRQVVUVpaamRtUp5ZvPghHxOpsrYRdARER//vOfKTMzk9RqNeXn51Nzc7PcJckGwB2ngwcPSm08Hg+9+uqrlJSURHFxcfT000+Tw+EI+pwrV67Qtm3bKDY2llJSUmjfvn3k8/nm+NfI59YA4nU2N3g4DsaYbMKqD4gxFlk4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJhgOIMSab/w8UeHgzMm8L9AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-1.2  -0.07], [0.6  0.07], (2,), float32)\n",
      "env.action_space= Box(-1.0, 1.0, (1,), float32)\n",
      "state= [-0.50329584  0.        ]\n",
      "action= [-0.03706472]\n",
      "next_state= [-5.0350362e-01 -2.0777507e-04]\n",
      "reward= -1.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ddpg.ddpg.DDPG at 0x7f28d98ad7c0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import DDPG\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = DDPG(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    buffer_size=10_10000,  #replay_buff_size\n",
    "    learning_starts=100,  #从N步以后开始训练\n",
    "    batch_size=100,  #一次采样数据量\n",
    "    tau=0.005,  #target网络每次软更新量\n",
    "    gamma=0.9,\n",
    "    train_freq=(20, 'step'),  #训练频率\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-400.0, 0.0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/6.DDPG.Mountain Car Continuous')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(34.9, 0.5385164807134505)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = DDPG.load('save/6.DDPG.Mountain Car Continuous')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhhklEQVR4nO3daXAUdf4/8HdP5swxuQ+SzBCUI0QgQM4hEQUikcUD9cGu5QPX3dJFw5aoZRWp+ou1VVsVa/fJrrsWPthd9cHWssWW7CpCNCSQg0xuIzkgigIZTGYSSGYmx2Su/vwfYObHCEgCSXqOz6uqq5zu78x8pkO/7f5297cFIiIwxpgEZFIXwBgLXxxAjDHJcAAxxiTDAcQYkwwHEGNMMhxAjDHJcAAxxiTDAcQYkwwHEGNMMhxAjDHJSBZA7733HrKysqBWq1FUVIS2tjapSmGMSUSSAPr3v/+N119/HW+//Ta6urqQm5uL8vJyjIyMSFEOY0wighQ3oxYVFaGgoAB//etfAQCiKEKn0+G3v/0tDhw4sNTlMMYkIl/qL3S5XOjs7ERlZaVvnkwmQ1lZGYxG4y3f43Q64XQ6fa9FUcTY2BgSExMhCMKi18wYmx8iwsTEBNLT0yGT3f5Aa8kD6OrVq/B6vUhNTfWbn5qaivPnz9/yPVVVVfjd7363FOUxxhaQyWRCZmbmbZcveQDdjcrKSrz++uu+1zabDXq9HiaTCVqtVsLKGGO3YrfbodPpEBMT85PtljyAkpKSEBERAYvF4jffYrEgLS3tlu9RqVRQqVQ3zddqtRxAjAWwO3WRLPlZMKVSiby8PNTW1vrmiaKI2tpaGAyGpS6HMSYhSQ7BXn/9dTz//PPIz89HYWEh/vSnP2FqagovvPCCFOUwxiQiSQD9/Oc/x+joKA4ePAiz2YyNGzeiurr6po5pxlhok+Q6oHtlt9sRGxsLm83GfUCMBaC5bqNBcRaMMbb4XK7v4fWOQ6HQISIiEoB80a+z4wBijAEArNb/Ymjo/0EuT4FanY3IyFxoNBugVj8AhSIVERGxmD1vtVDBxAHEGPuBCK/XCq/XCqfza9hsnwCQQyZTQ6nUQaVaiaioQmi15YiKKliQb+QAYozdhgxyeRzk8mXQaB7w7REpFOkL9g0cQIwxAIDbLYdavREazdofwmYTVKoVUCjSIZOpAcgWvE+IA4gxBgDo6krGli3/Q0JCBgQhYkm+k0dEZIwBANxuGQQhesnCB+AAYozh+u1QbrcbFy9exMcff4wDBw5gcHBw0b+XD8EYCzOz1x57vV5YLBb09vaisbERn332GUwmE8bHx0FEyMjIwL59+xb1WiAOIMbCABHB4/HAarWiv78fRqMRDQ0N6O3thdlshtvtvuk97e3tEEURERGLd0jGAcRYiCIi2Gw2fPfdd2hubkZjYyNaW1sxMjICh8Nxx/e3tbVhenr6jmP63AsOIMZClMViwZNPPone3l44HA7M97bPS5cuoa+vD8XFxYtUIXdCMxaytFot5HI5pqen5xQ+giAgOTkZOp0OWq0WLpcLXV1di1oj7wExFqI0Gg02btyI5ubmO7ZVKpVYtWoVHnnkEcTExMBut+PUqVNoaGjAb37zm0XrB+I9IMZClCAI2LZtm9+8qKgo3HfffcjKyoJGo/G1KykpQU5ODlQqFSIiIhAfH49du3ZhdHQUNptt0WrkPSDGQlhubi6SkpJw9epV6HQ6PP7440hMTAQRwWw245NPPoHdbkdubi6Ghob83qtWq/HLX/4SsbGxi1Yf7wExFsIyMjKwfPlyxMTE4PHHH0dycjIiIiIgl8uRkZGBxx57DAkJCVAqlTe9VxAExMbG8ml4xtjdUavVKC0txczMDJKTk/0uKhQEATqdDn//+9/hcrlw+fJlv/cqFArk5OQAgF8nNhHd9Dl3iwOIsRAmk8lQWlqKM2fO3DIoZDIZcnJykJSUhNbWVl8bIsKaNWswPj6OEydOYGZmBjabDRqNBpOTk9BoNFAoFFCr1UhISEB0dDQyMjIQExNzy72p2+EAYizElZWVITExEf39/bh69arfMp1O5zss0+v1GB8fx9TUFBITE6FQKCCKIlauXIn4+HgoFAq/xyyLooiZmRmMjY1heHgYFy9ehM1mQ1RUFOLj4+dUGw9Kz1gYICJcuXIFn332GUZHRyEIAlJTU5Gfn48rV67g6tWrUCgUKCwshF6v93sQ6FwOsW6MEavVioGBARgMhjtuoxxAjIUJIsL09DSuXLmCb775BiaTCcuXL0dubi5SUlIgly/cIPT8VAzGmA8Rwe124/z58+jt7UV2djYefvhhREdHS1oXBxBjIc7r9aKvrw89PT1Yu3Ytnn32WSgUikV/5M5ccAAxFqKICN9//z3a29uRlJSEPXv2IDIyMiCCZxYHEGMhyOVyoa2tDRaLBdu3b0dcXFxABc8sDiDGQggRYWhoCKdOnUJ+fj4MBsOiXsl8rziAGAsRXq8X3d3dsFgseOKJJxATExOQez03mve9YA0NDXj88ceRnp4OQRDw3//+1285EeHgwYNYtmwZNBoNysrK8M033/i1GRsbw3PPPQetVou4uDj8+te/xuTk5D39EMbC2dTUFD7//HMIgoDy8nJotdqADx/gLgJoamoKubm5eO+99265/A9/+APeffddvP/++2htbUVUVBTKy8sxMzPja/Pcc8+hr68PNTU1OHbsGBoaGvDSSy/d/a9gLEwREUwmE06ePIlNmzZh06ZNAX3IdRO6BwDo6NGjvteiKFJaWhr98Y9/9M2zWq2kUqnoX//6FxER9ff3EwBqb2/3tTlx4gQJgkDff//9nL7XZrMRALLZbPdSPmNBTRRFOn/+PP3nP/+hyclJEkVR6pJ85rqNLuhwHBcvXoTZbEZZWZlvXmxsLIqKimA0GgEARqMRcXFxyM/P97UpKyuDTCZDa2vrLT/X6XTCbrf7TYyFM4/HA6PRiNHRUTzxxBOIiooKikOuH1vQADKbzQCA1NRUv/mpqam+ZWazGSkpKX7L5XI5EhISfG1+rKqqCrGxsb5Jp9MtZNmMBRW3243GxkYoFAqUlJRAoVBIXdJdC4oBySorK2Gz2XyTyWSSuiTGlhwRYWZmBvX19dDr9cjPzw/KvZ4bLWgApaWlAbj+OJAbWSwW37K0tDSMjIz4Lfd4PBgbG/O1+TGVSgWtVus3MRZOiAhOpxM1NTVYs2YN7rvvvqAPH2CBA2jFihVIS0tDbW2tb57dbkdraysMBgMAwGAwwGq1orOz09emrq4OoiiiqKhoIcthLGRMTU2hrq4OxcXFyMzMDInwAe7iQsTJyUlcuHDB9/rixYvo7u5GQkIC9Ho99u/fj9///vdYtWoVVqxYgbfeegvp6enYs2cPAGDt2rV49NFH8eKLL+L999+H2+3Gvn378Itf/ALp6ekL9sMYCwVEhMnJSdTW1mLr1q1ISEiQuqSFNd/Ta6dOnSIAN03PP/88EV0/NfjWW29RamoqqVQq2rFjBw0MDPh9xrVr1+jZZ5+l6Oho0mq19MILL9DExMSca+DT8Cxc2O12OnHiBI2PjwfUafY7mes2ygOSMRaAiAhWqxVnzpxBaWkp4uLipC5pXua6jQbFWTDGwgn9MHLhmTNnYDAYFvW5XFLjm1EZCzAzMzM4ffo0tmzZEnp9Pj/Ce0CMBRCn04na2lrk5+fP+ckSwYwDiLEAMXuF88aNG5GSkhIyp9p/CgcQYwHA7XajtrYW9913HzIyMsIifAAOIMYkR0Roa2tDWloaVqxYETbhA3AAMSYpIkJPTw+USiVyc3PDKnwADiDGJENE6O3txdDQEPLy8sIufAAOIMYkMzw8jKGhIWzfvt3vmevhJDx/NWMSm70he+vWrVAqlVKXIxkOIMaWmMPhwPHjx2EwGKBWq6UuR1JBfSV0EN7GxsKc1+tFQ0MDHnroISQlJUldjuSCeg/IbrdzCLGgIYoijEYjMjMzeeiZHwR1ADU3N8PlckldBmN3REQ4d+4cXC4XcnJywvKM160EdQBlZ2ejtrYWHo9H6lIYuy0igsVigclkwoMPPsjhc4OgDqCsrCwkJSWhu7ubD8VYwJqcnITRaMTWrVuD+gkWiyGoA0gQBOTl5cFut+Py5cscQizguN1unD59GgUFBdBoNFKXE3CCOoAAICIiAqWlpejo6MD4+LjU5TDmI4oiWlpasHr16rC6wXQ+gj6AAEChUODhhx9GY2MjnE6n1OUw5rvNQiaTYdWqVRw+txESASQIAhITE7F+/Xo0NzdDFEWpS2JhjIgwOjqKixcvoqioKGxvs5iLkFkzgiBgxYoViIqKwpdffsn9QUwyDocDLS0t2L59O+TyoL7Wd9GFTAAB10No8+bNMJvNMJvNHEJsyXk8HtTV1WHz5s2IiYmRupyAF1IBBAByuRzbtm1Da2srHA6H1OWwMEJE6OjoQHJyMjIyMqQuJyiEXAABgEajQUFBAU6ePAm32y11OSwMEBGuXLmCiYkJ5Ofnc6fzHIVkAAmCgPT0dKSnp3N/EFsSdrsdbW1tePDBBxERESF1OUEjJAMIuB5CmzZtwsTEBAYHB6Uuh4Uwj8eD2tpaFBUVQaVSSV1OUAnZAAKuX6RYUlKCjo4OjI2N8Z4QW3Cz/T6rV69GZmYmH3rNU0gHEACoVCoYDAbU19fD6/VKXQ4LMd9++y2mp6eRk5MjdSlBaV4BVFVVhYKCAsTExCAlJQV79uzBwMCAX5uZmRlUVFQgMTER0dHReOaZZ2CxWPzaDA4OYvfu3YiMjERKSgrefPPNRbujXRAELFu2DKtWrUJHRwfvBbEFY7Va0dbWhuLiYr7Y8C7Na63V19ejoqICLS0tqKmpgdvtxs6dOzE1NeVr89prr+HTTz/FkSNHUF9fj6GhITz99NO+5V6vF7t374bL5UJzczM++ugjfPjhhzh48ODC/aofEQQBDzzwACYnJ/Htt99yCLF75na70dTUhEceeQSRkZFSlxO86B6MjIwQAKqvryciIqvVSgqFgo4cOeJrc+7cOQJARqORiIiOHz9OMpmMzGazr82hQ4dIq9WS0+mc0/fabDYCQDabbV71Tk9P0z//+U+y2+3zeh9jNxJFkc6cOUN9fX0kiqLU5QSkuW6j97TfaLPZAAAJCQkAgM7OTrjdbpSVlfnaZGdnQ6/Xw2g0AgCMRiPWr1+P1NRUX5vy8nLY7Xb09fXd8nucTifsdrvfdDc0Gg3KysrQ0NDAg5ixu0JE+Oabb3Dt2jVkZ2dzp/M9uusAEkUR+/fvR0lJCdatWwcAMJvNUCqViIuL82ubmpoKs9nsa3Nj+Mwun112K1VVVYiNjfVNOp3ubstGcnIydDod9wexuzIxMYH+/n488sgj3O+zAO56DVZUVKC3txeHDx9eyHpuqbKyEjabzTeZTKa7/ixBELBu3TqMjo7i0qVLHEJszjweDxoaGlBSUhL2j9NZKHcVQPv27cOxY8dw6tQpZGZm+uanpaXB5XLBarX6tbdYLEhLS/O1+fFZsdnXs21+TKVSQavV+k33QiaToaysDGfPnsXk5OQ9fRYLD0SEhoYGpKen8+N0FtC8AoiIsG/fPhw9ehR1dXVYsWKF3/K8vDwoFArU1tb65g0MDGBwcBAGgwEAYDAY0NPTg5GREV+bmpoaaLXaJb2WQqPRoKioCE1NTdwfxH4SEWFwcBAejwcbN27kfp+FNJ+e7ZdffpliY2Pp9OnTNDw87Jump6d9bfbu3Ut6vZ7q6uqoo6ODDAYDGQwG33KPx0Pr1q2jnTt3Und3N1VXV1NycjJVVlYueA/7nYiiSJ2dnXTmzBk+m8Fua2Jigj7++GNyOBxSlxI05rqNziuAANxy+uCDD3xtHA4HvfLKKxQfH0+RkZH01FNP0fDwsN/nXLp0iXbt2kUajYaSkpLojTfeILfbveA/bi48Hg998cUXNDg4eM+fxUKPy+WiI0eO0NDQkNSlBJW5bqMCUfD1wtrtdsTGxsJms91zfxBwfQS76upq7Ny5E5GRkbyLzQBcP/Tq7Oz0DXTH/y7mbq7bKJ9HBKBWq1FYWIiTJ0/yeNLM58qVKxgfH+d+n0XEAYT/Gz8oMzMTX331FZ+aZ7Db7WhtbUVJSQmP77OIOIB+IAgCNm7cCIvFguHhYanLYRLyer1oampCcXExP0xwkXEA3SAiIgJbt25FU1OT3w22LHwQEbq7u7Fs2TJ+mOAS4AD6kcjISN/1QdwfFF6ICCaTCaOjo9iwYQOHzxLgAPoRQRCg1+uRmJiInp4e7g8KIw6HA62trTyu8xLiALqF2fGkr1y5gu+//55DKAx4vV40NDRgy5YtiIqKkrqcsMEBdBsREREoLS1FW1sbP18sxBERvvrqK6SkpCA9PV3qcsIKB9BPiI2NRUFBAZqamng86RBFRBgaGsKVK1eQm5vL/T5LjAPoDjIzM5GYmIiuri4+FAtBDocD7e3t2LFjB4/vIwFe43cgCAJyc3NhNpsxNDTEIRRCPB4PGhsbUVhYiKioKN77kQAH0BzI5XLs2LGD+4NCCP3wPK/ExEQsW7ZM6nLCFgfQHM2OH3T69GkePygEmEwmTE5OYtOmTbznIyEOoDmafb5YcnIyP28+iBERJiYm0NXVhdLSUr7eR2IcQPMwOyyD3W7n580HKa/Xi+PHj6O4uJif4x4AOIDmafZ5893d3RgfH+c9oSAiiiLa2tqwfv16pKWl8aFXAOAAugsqlQolJSWor6/n/qAgQUS4cOECPB4PsrOzpS6H/YAD6C4IgoDExEQ88MADMBqNfNNqgCMijIyMoKOjg5/jHmD4L3GXBEHAypUrERERgb6+Pj4UC2BOpxMtLS147LHHoFQqpS6H3YAD6B4IgoDCwkJ8/fXXPIhZgPJ6vWhsbMTmzZsRExMjdTnsRziA7pFCocCjjz6K9vZ2fshhgJkdVD4uLg6ZmZnc6RyAOIAWQGRkJAwGA+rr6+F2u6Uuh+F6+Hz33XdwOBz8RIsAxgG0AARBQHJyMrKystDS0sKd0hIjIly7dg39/f3YsmULX2wYwDiAFoggCMjJyYFKpeJOaYk5HA58/vnnKC0thVwul7oc9hM4gBaQIAjIz8+HyWTCpUuXOIQk4PF40NDQgO3btyM+Pp4PvQIcB9ACk8lkeOihh/Dll1/yldJLTBRFNDc3IysrC2lpaVKXw+aAA2gRREVFoaysDE1NTZiZmZG6nLBAROjp6UFERATWrFnDez5BYl4BdOjQIWzYsAFarRZarRYGgwEnTpzwLZ+ZmUFFRQUSExMRHR2NZ555BhaLxe8zBgcHsXv3bkRGRiIlJQVvvvlmSN7OEBMTg7y8PFRXV8PlckldTkgjIly6dAljY2MoLi7m8Aki8wqgzMxMvPPOO+js7ERHRwe2b9+OJ598En19fQCA1157DZ9++imOHDmC+vp6DA0N4emnn/a93+v1Yvfu3XC5XGhubsZHH32EDz/8EAcPHlzYXxUAZh/3vHbtWjQ3N/OZsUVCRDCbzejr6+PHKAcjukfx8fH0t7/9jaxWKykUCjpy5Ihv2blz5wgAGY1GIiI6fvw4yWQyMpvNvjaHDh0irVZLTqdzzt9ps9kIANlstnstf9GJokhdXV3U0dFBoihKXU7Isdvt9L///Y8mJyelLoXdYK7b6F33AXm9Xhw+fBhTU1MwGAzo7OyE2+1GWVmZr012djb0ej2MRiMAwGg0Yv369UhNTfW1KS8vh91u9+1FhZrZMaXHxsZw9uxZ7pReQNPT02hsbMSDDz7Iz/IKUvMOoJ6eHkRHR0OlUmHv3r04evQocnJyYDaboVQqERcX59c+NTUVZrMZAGA2m/3CZ3b57LLbcTqdsNvtflMwkclk2LZtG8bGxvj0/AJxOp2orq5Gbm7uTf/mWPCYdwCtWbMG3d3daG1txcsvv4znn38e/f39i1GbT1VVFWJjY32TTqdb1O9bDHK5HKWlpTh37hzMZjOH0D3wer1oamrC5s2bkZGRwZ3OQWzeAaRUKrFy5Urk5eWhqqoKubm5+POf/4y0tDS4XC5YrVa/9haLxXdNRlpa2k1nxWZf/9R1G5WVlbDZbL7JZDLNt+yAoFAo8PDDD6OjowPXrl2TupygNBs+mZmZWL58udTlsHt0z9cBiaIIp9OJvLw8KBQK1NbW+pYNDAxgcHAQBoMBAGAwGNDT04ORkRFfm5qaGmi1WuTk5Nz2O1Qqle/U/+wUrCIjI7Ft2za0t7djfHxc6nKCiiiK6OjogEajwerVq3nPJxTMp2f7wIEDVF9fTxcvXqSzZ8/SgQMHSBAE+uKLL4iIaO/evaTX66muro46OjrIYDCQwWDwvd/j8dC6deto586d1N3dTdXV1ZScnEyVlZWL0sMeqERRJKvVSocPH6axsTE+OzYHXq+XOjo6qLu7m7xer9TlsDuY6zY6rwD61a9+RcuXLyelUknJycm0Y8cOX/gQETkcDnrllVcoPj6eIiMj6amnnqLh4WG/z7h06RLt2rWLNBoNJSUl0RtvvEFut3s+ZQR9AM26du0aVVdXk91ul7qUgOb1eqm5uZnOnDnD4RMk5rqNCkTB1xtqt9sRGxsLm80W1IdjADA+Po7GxkaUlpbyzZO3IIoienp64HK5kJeXx+M5B4m5bqP815RYXFwcDAYDmpubMTExIXU5AUUURbS0tMDtdnP4hCj+i0psdjCzkpISNDc3Y2xsjE/R43r4dHV1AQA2b97M4ROi+K8aIOLi4lBUVISTJ0+G/Sl6r9eLlpYWKBQKfoxOiOO/bIAQBAHx8fHYtWsX2tvbMTw8HJZ7Qm63G/X19VCpVNiwYQOHT4jjv26AiY6OxrZt29DV1YULFy6EVQhNTU2hrq4OWVlZPJB8mOAACjCCIECtVmPnzp0YHh7G2bNnQ34oDyKCzWbDiRMnsHbtWqxYsYLDJ0xwAAUohUKBkpISTE9P4/Tp0yH7uB8iwuXLl1FfX4/t27dDr9dz+IQRDqAAFhERgeLiYuj1etTU1MBut4fUIZkoiujt7UVnZyd27tyJ+Ph4qUtiS4wDKMAJgoD7778f+fn5qK6uhslkCvoQIiLMzMygrq4ODocDTz75JNRqNe/5hCEOoCAwe63QE088gYGBATQ1NQXtONpEhJGRERw7dgzp6ekoKCjgZ3eFMQ6gIDHbOb1jxw6o1Wp88sknuHr1alDtDbndbjQ2NqK1tRXl5eVYu3Yt7/WEOf5fT5CRyWQoKCjAypUrUVdXh5SUFBQXF0MulwfsxkxEGB4eRltbGzIzM3nweObDN6MGKSKC1+vF2bNncfHiReTn50On0wXchXvT09Nob2/H0NAQdu3ahdjY2IANSrZw5rqNcgAFOSLCxMQEmpqaIIoiiouLkZiYKOlGTkTweDz46quv8PXXXyM/Px/3338/7/WEEQ6gMDPbuVtXV4eYmBgUFRUhKSlpyYPI6XSip6cHFy5cwP3334/169dDpVLxXk+Y4QAKU16v19ff4vF4UFhYiIyMjEXtI5rdC+vp6YHJZEJmZiY2bdqEyMhIDp4wxQEUxuj6SJcYHR317Y2sXbsWq1evRkpKCmQy2T0Fw+w/GYfDgQsXLqC/vx8ymQyrVq3CAw88AIVCwcET5jiAGIDrYeF0OjE4OIju7m44HA4kJSUhPT0dy5cvh0aj8R0i3S40Zvt0nE4nxsbGcPnyZVy4cAFKpRI6nQ7Z2dlISkoKuA5wJh0OIHYTIoLL5cLIyAguXLiA8fFxmM1mREdHQxAExMTEQK1Ww+PxQK1Ww2q1QiaTYWpqCqIoQq1WIyoqCqtWrUJycjLi4uI4dNgtzXUb5euAwoggCFCpVNDpdNDpdCAiiKIIURQxMTEBh8MBt9sNp9MJpVIJIoJWq0V0dLTvsOqn9pQYmy8OoDAmCAIiIiIQERGBhIQEqcthYYj3nxljkuEAYoxJhgOIMSYZDiDGmGQ4gBhjkuEAYoxJhgOIMSYZDiDGmGQ4gBhjkuEAYoxJJihvxZi9f9Zut0tcCWPsVma3zTvd6x6UAXTt2jUAgE6nk7gSxthPmZiYQGxs7G2XB2UAzd44OTg4+JM/jvmz2+3Q6XQwmUw8jMkc8Tq7O7OjZKanp/9ku6AMoNkxaGJjY/kfxV3QarW83uaJ19n8zWXngDuhGWOS4QBijEkmKANIpVLh7bffhkqlkrqUoMLrbf54nS2uoBwTmjEWGoJyD4gxFho4gBhjkuEAYoxJhgOIMSaZoAyg9957D1lZWVCr1SgqKkJbW5vUJUmmqqoKBQUFiImJQUpKCvbs2YOBgQG/NjMzM6ioqEBiYiKio6PxzDPPwGKx+LUZHBzE7t27ERkZiZSUFLz55pvweDxL+VMk884770AQBOzfv983j9fZEqEgc/jwYVIqlfSPf/yD+vr66MUXX6S4uDiyWCxSlyaJ8vJy+uCDD6i3t5e6u7vpZz/7Gen1epqcnPS12bt3L+l0OqqtraWOjg4qLi6mLVu2+JZ7PB5at24dlZWV0ZdffknHjx+npKQkqqyslOInLam2tjbKysqiDRs20Kuvvuqbz+tsaQRdABUWFlJFRYXvtdfrpfT0dKqqqpKwqsAxMjJCAKi+vp6IiKxWKykUCjpy5Iivzblz5wgAGY1GIiI6fvw4yWQyMpvNvjaHDh0irVZLTqdzaX/AEpqYmKBVq1ZRTU0NPfTQQ74A4nW2dILqEMzlcqGzsxNlZWW+eTKZDGVlZTAajRJWFjhsNhuA/7tht7OzE26322+dZWdnQ6/X+9aZ0WjE+vXrkZqa6mtTXl4Ou92Ovr6+Jax+aVVUVGD37t1+6wbgdbaUgupm1KtXr8Lr9fr90QEgNTUV58+fl6iqwCGKIvbv34+SkhKsW7cOAGA2m6FUKhEXF+fXNjU1FWaz2dfmVut0dlkoOnz4MLq6utDe3n7TMl5nSyeoAoj9tIqKCvT29qKpqUnqUgKayWTCq6++ipqaGqjVaqnLCWtBdQiWlJSEiIiIm85GWCwWpKWlSVRVYNi3bx+OHTuGU6dOITMz0zc/LS0NLpcLVqvVr/2N6ywtLe2W63R2Wajp7OzEyMgINm/eDLlcDrlcjvr6erz77ruQy+VITU3ldbZEgiqAlEol8vLyUFtb65sniiJqa2thMBgkrEw6RIR9+/bh6NGjqKurw4oVK/yW5+XlQaFQ+K2zgYEBDA4O+taZwWBAT08PRkZGfG1qamqg1WqRk5OzND9kCe3YsQM9PT3o7u72Tfn5+Xjuued8/83rbIlI3Qs+X4cPHyaVSkUffvgh9ff300svvURxcXF+ZyPCycsvv0yxsbF0+vRpGh4e9k3T09O+Nnv37iW9Xk91dXXU0dFBBoOBDAaDb/nsKeWdO3dSd3c3VVdXU3JyclidUr7xLBgRr7OlEnQBRET0l7/8hfR6PSmVSiosLKSWlhapS5IMgFtOH3zwga+Nw+GgV155heLj4ykyMpKeeuopGh4e9vucS5cu0a5du0ij0VBSUhK98cYb5Ha7l/jXSOfHAcTrbGnwcByMMckEVR8QYyy0cAAxxiTDAcQYkwwHEGNMMhxAjDHJcAAxxiTDAcQYkwwHEGNMMhxAjDHJcAAxxiTDAcQYkwwHEGNMMv8fIQ0Zdm1dTXcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35.0 66 [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 100]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
